--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
meshchatx/src/backend/database/misc.py 147e9451330dfa57b06aa877aa88941b226588af (147e9451) Text, 7.99 KB
Tff7b72from T7ee787datetime Tff7b72import Te6edf3UTCTb4b4b4, Te6edf3datetime
Tff7b72from T7ee787.T7ee787provider Tff7b72import Te6edf3DatabaseProvider
Tff7b72class T56d364MiscDAOTb4b4b4:
Tff7b72def Tff7b72__init__Tb4b4b4(Tff7b72selfTb4b4b4, Te6edf3providerTb4b4b4: Te6edf3DatabaseProviderTb4b4b4)Tb4b4b4:
Tff7b72selfTff7b72.Td2a8ffprovider Tff7b72= Te6edf3provider
T8b949e# Blocked Destinations
Tff7b72def Td2a8ffadd_blocked_destinationTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3destination_hashTb4b4b4)Tb4b4b4:
Te6edf3now Tff7b72= Te6edf3datetimeTff7b72.Td2a8ffnowTb4b4b4(Te6edf3UTCTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(
Ta5d6ff"Ta5d6ffINSERT OR IGNORE INTO blocked_destinations (destination_hash, created_at, updated_at) VALUES (?, ?, ?)Ta5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3destination_hashTb4b4b4, Te6edf3nowTb4b4b4, Te6edf3nowTb4b4b4)Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8ffis_destination_blockedTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3destination_hashTb4b4b4)Tb4b4b4:
Tff7b72return Tb4b4b4(
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8fffetchoneTb4b4b4(
Ta5d6ff"Ta5d6ffSELECT 1 FROM blocked_destinations WHERE destination_hash = ?Ta5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3destination_hashTb4b4b4,Tb4b4b4)Tb4b4b4,
Tb4b4b4)
Tff7b72is Tff7b72not Tff7b72None
Tb4b4b4)
Tff7b72def Td2a8ffget_blocked_destinationsTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Tff7b72return Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8fffetchallTb4b4b4(Ta5d6ff"Ta5d6ffSELECT * FROM blocked_destinationsTa5d6ff"Tb4b4b4)
Tff7b72def Td2a8ffdelete_blocked_destinationTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3destination_hashTb4b4b4)Tb4b4b4:
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(
Ta5d6ff"Ta5d6ffDELETE FROM blocked_destinations WHERE destination_hash = ?Ta5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3destination_hashTb4b4b4,Tb4b4b4)Tb4b4b4,
Tb4b4b4)
T8b949e# Spam Keywords
Tff7b72def Td2a8ffadd_spam_keywordTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3keywordTb4b4b4)Tb4b4b4:
Te6edf3now Tff7b72= Te6edf3datetimeTff7b72.Td2a8ffnowTb4b4b4(Te6edf3UTCTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(
Ta5d6ff"Ta5d6ffINSERT OR IGNORE INTO spam_keywords (keyword, created_at, updated_at) VALUES (?, ?, ?)Ta5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3keywordTb4b4b4, Te6edf3nowTb4b4b4, Te6edf3nowTb4b4b4)Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8ffget_spam_keywordsTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Tff7b72return Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8fffetchallTb4b4b4(Ta5d6ff"Ta5d6ffSELECT * FROM spam_keywordsTa5d6ff"Tb4b4b4)
Tff7b72def Td2a8ffdelete_spam_keywordTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3keyword_idTb4b4b4)Tb4b4b4:
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(Ta5d6ff"Ta5d6ffDELETE FROM spam_keywords WHERE id = ?Ta5d6ff"Tb4b4b4, Tb4b4b4(Te6edf3keyword_idTb4b4b4,Tb4b4b4)Tb4b4b4)
Tff7b72def Td2a8ffcheck_spam_keywordsTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3titleTb4b4b4, Te6edf3contentTb4b4b4)Tb4b4b4:
Te6edf3keywords Tff7b72= Tff7b72selfTff7b72.Td2a8ffget_spam_keywordsTb4b4b4(Tb4b4b4)
Te6edf3search_text Tff7b72= Tb4b4b4(Te6edf3title Tff7b72+ Ta5d6ff"Ta5d6ff Ta5d6ff" Tff7b72+ Te6edf3contentTb4b4b4)Tff7b72.Td2a8fflowerTb4b4b4(Tb4b4b4)
Tff7b72for Te6edf3kw Tff7b72in Te6edf3keywordsTb4b4b4:
Tff7b72if Te6edf3kwTb4b4b4[Ta5d6ff"Ta5d6ffkeywordTa5d6ff"Tb4b4b4]Tff7b72.Td2a8fflowerTb4b4b4(Tb4b4b4) Tff7b72in Te6edf3search_textTb4b4b4:
Tff7b72return Tff7b72True
Tff7b72return Tff7b72False
T8b949e# User Icons
Tff7b72def Td2a8ffupdate_lxmf_user_iconTb4b4b4(
Tff7b72selfTb4b4b4,
Te6edf3destination_hashTb4b4b4,
Te6edf3icon_nameTb4b4b4,
Te6edf3foreground_colourTb4b4b4,
Te6edf3background_colourTb4b4b4,
Tb4b4b4)Tb4b4b4:
Te6edf3now Tff7b72= Te6edf3datetimeTff7b72.Td2a8ffnowTb4b4b4(Te6edf3UTCTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(
T8b949e"""
INSERT INTO lxmf_user_icons (destination_hash, icon_name, foreground_colour, background_colour, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(destination_hash) DO UPDATE SET
icon_name = EXCLUDED.icon_name,
foreground_colour = EXCLUDED.foreground_colour,
background_colour = EXCLUDED.background_colour,
updated_at = EXCLUDED.updated_at
"""Tb4b4b4,
Tb4b4b4(
Te6edf3destination_hashTb4b4b4,
Te6edf3icon_nameTb4b4b4,
Te6edf3foreground_colourTb4b4b4,
Te6edf3background_colourTb4b4b4,
Te6edf3nowTb4b4b4,
Te6edf3nowTb4b4b4,
Tb4b4b4)Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8ffget_user_iconTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3destination_hashTb4b4b4)Tb4b4b4:
Tff7b72return Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8fffetchoneTb4b4b4(
Ta5d6ff"Ta5d6ffSELECT * FROM lxmf_user_icons WHERE destination_hash = ?Ta5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3destination_hashTb4b4b4,Tb4b4b4)Tb4b4b4,
Tb4b4b4)
T8b949e# Forwarding Rules
Tff7b72def Td2a8ffget_forwarding_rulesTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3identity_hashTff7b72=Tff7b72NoneTb4b4b4, Te6edf3active_onlyTff7b72=Tff7b72FalseTb4b4b4)Tb4b4b4:
Te6edf3query Tff7b72= Ta5d6ff"Ta5d6ffSELECT * FROM lxmf_forwarding_rules WHERE 1=1Ta5d6ff"
Te6edf3params Tff7b72= Tb4b4b4[Tb4b4b4]
Tff7b72if Te6edf3identity_hashTb4b4b4:
Te6edf3query Tff7b72+Tff7b72= Ta5d6ff"Ta5d6ff AND (identity_hash = ? OR identity_hash IS NULL)Ta5d6ff"
Te6edf3paramsTff7b72.Td2a8ffappendTb4b4b4(Te6edf3identity_hashTb4b4b4)
Tff7b72if Te6edf3active_onlyTb4b4b4:
Te6edf3query Tff7b72+Tff7b72= Ta5d6ff"Ta5d6ff AND is_active = 1Ta5d6ff"
Tff7b72return Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8fffetchallTb4b4b4(Te6edf3queryTb4b4b4, Te6edf3paramsTb4b4b4)
Tff7b72def Td2a8ffcreate_forwarding_ruleTb4b4b4(
Tff7b72selfTb4b4b4,
Te6edf3identity_hashTb4b4b4,
Te6edf3forward_to_hashTb4b4b4,
Te6edf3source_filter_hashTb4b4b4,
Te6edf3is_activeTff7b72=Tff7b72TrueTb4b4b4,
Te6edf3nameTff7b72=Tff7b72NoneTb4b4b4,
Tb4b4b4)Tb4b4b4:
Te6edf3now Tff7b72= Te6edf3datetimeTff7b72.Td2a8ffnowTb4b4b4(Te6edf3UTCTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(
Ta5d6ff"Ta5d6ffINSERT INTO lxmf_forwarding_rules (identity_hash, forward_to_hash, source_filter_hash, is_active, name, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)Ta5d6ff"Tb4b4b4,
Tb4b4b4(
Te6edf3identity_hashTb4b4b4,
Te6edf3forward_to_hashTb4b4b4,
Te6edf3source_filter_hashTb4b4b4,
T79c0ff1 Tff7b72if Te6edf3is_active Tff7b72else T79c0ff0Tb4b4b4,
Te6edf3nameTb4b4b4,
Te6edf3nowTb4b4b4,
Te6edf3nowTb4b4b4,
Tb4b4b4)Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8ffdelete_forwarding_ruleTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3rule_idTb4b4b4)Tb4b4b4:
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(
Ta5d6ff"Ta5d6ffDELETE FROM lxmf_forwarding_rules WHERE id = ?Ta5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3rule_idTb4b4b4,Tb4b4b4)Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8fftoggle_forwarding_ruleTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3rule_idTb4b4b4)Tb4b4b4:
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(
Ta5d6ff"Ta5d6ffUPDATE lxmf_forwarding_rules SET is_active = NOT is_active WHERE id = ?Ta5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3rule_idTb4b4b4,Tb4b4b4)Tb4b4b4,
Tb4b4b4)
T8b949e# Archived Pages
Tff7b72def Td2a8ffarchive_pageTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3destination_hashTb4b4b4, Te6edf3page_pathTb4b4b4, Te6edf3contentTb4b4b4, Te6edf3page_hashTb4b4b4)Tb4b4b4:
Te6edf3now Tff7b72= Te6edf3datetimeTff7b72.Td2a8ffnowTb4b4b4(Te6edf3UTCTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(
Ta5d6ff"Ta5d6ffINSERT INTO archived_pages (destination_hash, page_path, content, hash, created_at) VALUES (?, ?, ?, ?, ?)Ta5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3destination_hashTb4b4b4, Te6edf3page_pathTb4b4b4, Te6edf3contentTb4b4b4, Te6edf3page_hashTb4b4b4, Te6edf3nowTb4b4b4)Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8ffget_archived_page_versionsTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3destination_hashTb4b4b4, Te6edf3page_pathTb4b4b4)Tb4b4b4:
Tff7b72return Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8fffetchallTb4b4b4(
Ta5d6ff"Ta5d6ffSELECT * FROM archived_pages WHERE destination_hash = ? AND page_path = ? ORDER BY created_at DESCTa5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3destination_hashTb4b4b4, Te6edf3page_pathTb4b4b4)Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8ffget_archived_pages_paginatedTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3destination_hashTff7b72=Tff7b72NoneTb4b4b4, Te6edf3queryTff7b72=Tff7b72NoneTb4b4b4)Tb4b4b4:
Te6edf3sql Tff7b72= Ta5d6ff"Ta5d6ffSELECT * FROM archived_pages WHERE 1=1Ta5d6ff"
Te6edf3params Tff7b72= Tb4b4b4[Tb4b4b4]
Tff7b72if Te6edf3destination_hashTb4b4b4:
Te6edf3sql Tff7b72+Tff7b72= Ta5d6ff"Ta5d6ff AND destination_hash = ?Ta5d6ff"
Te6edf3paramsTff7b72.Td2a8ffappendTb4b4b4(Te6edf3destination_hashTb4b4b4)
Tff7b72if Te6edf3queryTb4b4b4:
Te6edf3like_term Tff7b72= Ta5d6fffTa5d6ff"Ta5d6ff%Tffd700{Te6edf3queryTffd700}Ta5d6ff%Ta5d6ff"
Te6edf3sql Tff7b72+Tff7b72= Tb4b4b4(
Ta5d6ff"Ta5d6ff AND (destination_hash LIKE ? OR page_path LIKE ? OR content LIKE ?)Ta5d6ff"
Tb4b4b4)
Te6edf3paramsTff7b72.Td2a8ffextendTb4b4b4(Tb4b4b4[Te6edf3like_termTb4b4b4, Te6edf3like_termTb4b4b4, Te6edf3like_termTb4b4b4]Tb4b4b4)
Te6edf3sql Tff7b72+Tff7b72= Ta5d6ff"Ta5d6ff ORDER BY created_at DESCTa5d6ff"
Tff7b72return Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8fffetchallTb4b4b4(Te6edf3sqlTb4b4b4, Te6edf3paramsTb4b4b4)
Tff7b72def Td2a8ffdelete_archived_pagesTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3destination_hashTff7b72=Tff7b72NoneTb4b4b4, Te6edf3page_pathTff7b72=Tff7b72NoneTb4b4b4)Tb4b4b4:
Tff7b72if Te6edf3destination_hash Tff7b72and Te6edf3page_pathTb4b4b4:
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(
Ta5d6ff"Ta5d6ffDELETE FROM archived_pages WHERE destination_hash = ? AND page_path = ?Ta5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3destination_hashTb4b4b4, Te6edf3page_pathTb4b4b4)Tb4b4b4,
Tb4b4b4)
Tff7b72elseTb4b4b4:
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(Ta5d6ff"Ta5d6ffDELETE FROM archived_pagesTa5d6ff"Tb4b4b4)
T8b949e# Crawl Tasks
Tff7b72def Td2a8ffupsert_crawl_taskTb4b4b4(
Tff7b72selfTb4b4b4,
Te6edf3destination_hashTb4b4b4,
Te6edf3page_pathTb4b4b4,
Te6edf3statusTff7b72=Ta5d6ff"Ta5d6ffpendingTa5d6ff"Tb4b4b4,
Te6edf3retry_countTff7b72=T79c0ff0Tb4b4b4,
Tb4b4b4)Tb4b4b4:
Te6edf3now Tff7b72= Te6edf3datetimeTff7b72.Td2a8ffnowTb4b4b4(Te6edf3UTCTb4b4b4)
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(
T8b949e"""
INSERT INTO crawl_tasks (destination_hash, page_path, status, retry_count, created_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(destination_hash, page_path) DO UPDATE SET
status = EXCLUDED.status,
retry_count = EXCLUDED.retry_count
"""Tb4b4b4,
Tb4b4b4(Te6edf3destination_hashTb4b4b4, Te6edf3page_pathTb4b4b4, Te6edf3statusTb4b4b4, Te6edf3retry_countTb4b4b4, Te6edf3nowTb4b4b4)Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8ffget_pending_crawl_tasksTb4b4b4(Tff7b72selfTb4b4b4)Tb4b4b4:
Tff7b72return Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8fffetchallTb4b4b4(
Ta5d6ff"Ta5d6ffSELECT * FROM crawl_tasks WHERE status = Ta5d6ff'Ta5d6ffpendingTa5d6ff'Ta5d6ff"Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8ffupdate_crawl_taskTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3task_idTb4b4b4, Tff7b72*Tff7b72*Te6edf3kwargsTb4b4b4)Tb4b4b4:
Te6edf3allowed_keys Tff7b72= Tb4b4b4{
Ta5d6ff"Ta5d6ffdestination_hashTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffpage_pathTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffstatusTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffretry_countTa5d6ff"Tb4b4b4,
Ta5d6ff"Ta5d6ffupdated_atTa5d6ff"Tb4b4b4,
Tb4b4b4}
Te6edf3filtered_kwargs Tff7b72= Tb4b4b4{Te6edf3kTb4b4b4: Te6edf3v Tff7b72for Te6edf3kTb4b4b4, Te6edf3v Tff7b72in Te6edf3kwargsTff7b72.Td2a8ffitemsTb4b4b4(Tb4b4b4) Tff7b72if Te6edf3k Tff7b72in Te6edf3allowed_keysTb4b4b4}
Tff7b72if Tff7b72not Te6edf3filtered_kwargsTb4b4b4:
Tff7b72return
Te6edf3set_clause Tff7b72= Ta5d6ff"Ta5d6ff, Ta5d6ff"Tff7b72.Td2a8ffjoinTb4b4b4(Tb4b4b4[Ta5d6fffTa5d6ff"Tffd700{Te6edf3kTffd700}Ta5d6ff = ?Ta5d6ff" Tff7b72for Te6edf3k Tff7b72in Te6edf3filtered_kwargsTb4b4b4]Tb4b4b4)
Te6edf3params Tff7b72= Tffa657listTb4b4b4(Te6edf3filtered_kwargsTff7b72.Td2a8ffvaluesTb4b4b4(Tb4b4b4)Tb4b4b4)
Te6edf3paramsTff7b72.Td2a8ffappendTb4b4b4(Te6edf3task_idTb4b4b4)
Te6edf3query Tff7b72= Ta5d6fffTa5d6ff"Ta5d6ffUPDATE crawl_tasks SET Tffd700{Te6edf3set_clauseTffd700}Ta5d6ff WHERE id = ?Ta5d6ff" T8b949e# noqa: S608
Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8ffexecuteTb4b4b4(Te6edf3queryTb4b4b4, Te6edf3paramsTb4b4b4)
Tff7b72def Td2a8ffget_pending_or_failed_crawl_tasksTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3max_retriesTb4b4b4, Te6edf3max_concurrentTb4b4b4)Tb4b4b4:
Tff7b72return Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8fffetchallTb4b4b4(
Ta5d6ff"Ta5d6ffSELECT * FROM crawl_tasks WHERE status IN (Ta5d6ff'Ta5d6ffpendingTa5d6ff'Ta5d6ff, Ta5d6ff'Ta5d6fffailedTa5d6ff'Ta5d6ff) AND retry_count < ? LIMIT ?Ta5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3max_retriesTb4b4b4, Te6edf3max_concurrentTb4b4b4)Tb4b4b4,
Tb4b4b4)
Tff7b72def Td2a8ffget_archived_page_by_idTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3archive_idTb4b4b4)Tb4b4b4:
Tff7b72return Tff7b72selfTff7b72.Td2a8ffproviderTff7b72.Td2a8fffetchoneTb4b4b4(
Ta5d6ff"Ta5d6ffSELECT * FROM archived_pages WHERE id = ?Ta5d6ff"Tb4b4b4,
Tb4b4b4(Te6edf3archive_idTb4b4b4,Tb4b4b4)Tb4b4b4,
Tb4b4b4)
──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────